#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <regex>
#include <string.h>
#include "../include/web/PageLibPreprocessor.hh"
#include "../include/SplitToolCppJieba.hh"

using namespace std;





int main(){
    Configuration conf;
    SplitToolCppJieba jieba(conf);
    PageLibPreprocessor preprocessor(conf, &jieba);

    preprocessor.doProcess();
    preprocessor.cutRedundantPages();
    preprocessor.bulidInvertIndexTable();
    preprocessor.storeOnDisk();
    return 0;
}